import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.URL;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;


public class WebReader {
	private static BufferedReader reader; //Useful for file
	private static ArrayList<String> SeedURLs = new ArrayList<String>(); //Useful for file
	private static ResultSet rsUserQuery = null;

	
	  public static void main(String[] args) throws Exception {
		  
/*	    //  Check to make sure only a single file argument was provided
	    if (args.length > 1 || args.length < 1)
	            System.out.println ("Too many/few arguments.");
	    
	    else if (args.length > 0){
	    	readFile(args[0]);                
	        try {
	                reader.close();
	        } catch (IOException e) {
	                e.printStackTrace();
	        }
	        //fetchTweetsByUser();
	        
	       }  */  // Use this code to fetch url from file
	       getData();
	    
	  }
//Use the following code to fetch url from a FILE  
/*	private static void readFile(String file){
		String input = null;
		try{
			reader = new BufferedReader (new FileReader (file));
			input = reader.readLine();
			while (input != null){
					processURLs(input);
					input = reader.readLine();				
				}
			}
			catch (Exception e){
				System.err.println ("Error reading from file " + file + ": " + e.getMessage());
			}	
	    }
	
	private static void processURLs(String input){
		//Add the read content to our Array List of strings
		SeedURLs.add(input);
		//System.out.format("Locations Size %s", SeedURLs.toArray());
	}   
	
	private static void fetchTweetsByUser() throws Exception {

	  	for(int i=0; i<SeedURLs.size();i++) {
	         //System.out.format("Querying %s times", SeedURLs.size());
	  		 //System.out.format("%s ", SeedURLs.get(i));
	         getData(SeedURLs.get(i));
	  	}   
	}
	
	static void getData(String address) throws Exception { */
	private static int id = 7453;
    static void getData() throws Exception {

	    try { 
            String url = "jdbc:sqlserver://127.0.0.1:1433;database=tversecrawler;integratedSecurity=true;"; 
    		String username = "tversecrawler";
    		String password = "Firehawk03";
    		
            Connection connect = DriverManager.getConnection(url,username,password); 
                        
			PreparedStatement stmtCheck = (PreparedStatement) connect.prepareStatement("select tweet_id, tempurl, id from url_temp3 where id > '" + id + "';");
			rsUserQuery = stmtCheck.executeQuery();
	           
			while (rsUserQuery.next()){
				
					String tweetid = rsUserQuery.getString("tweet_id");	
					String tiny_url = rsUserQuery.getString("tempurl");
					id = rsUserQuery.getInt("id");
					System.out.println("tweetid : " + tweetid);
					System.out.println("id : " + id);
					//System.out.println(id);


            //Fetch Title from URL
		    //URL page = new URL(tiny_url);
		    //HttpURLConnection conn = (HttpURLConnection) page.openConnection();
		    //conn.connect();
		    Document doc = Jsoup.connect(tiny_url)
		    					.cookie("auth", "token")
		    					.ignoreContentType(true)
		    					.timeout(100000)
		    					.ignoreHttpErrors(true)
		    					.post();
		    					//.get();

			String title = doc.title();
			String actualurl = doc.baseUri();
			
			System.out.println(tiny_url);
			System.out.println(title);	
		    System.out.println(actualurl); 
            //Insert our data into the database
 			Statement st = connect.createStatement(); 
            String stmt = "insert into twt_url_title(tweet_id, tiny_url, title, actual_url)" +
	            			" values (" + 
	            			"'"+tweetid+"',"+
	            			"'"+tiny_url+"',"+
	            			//"'"+title+"',"+
	            			"'"+title.replace("'", ".")+"',"+	 
	                        "'"+actualurl+"')";
                        
            System.out.println("SQL: " + stmt);
			
            try	{
                 	st = connect.createStatement();
                 	st.execute(stmt);
            	} catch (SQLException ex) {
                 	System.out.println(ex); }  
            }
            connect.close(); 
            } catch (Exception e) { 
            	System.err.println("Got an exception! "); 
            	System.err.println(e.getMessage()); 
            } 
            finally{
                       System.out.println("Calling getData() to continue ");
                       getData();    //method call to continue program
            	}
	  }	
}



//StringBuffer text = new StringBuffer();
//String text1 = null;
//InputStreamReader in = new InputStreamReader((InputStream) conn.getContent());
//BufferedReader buff = new BufferedReader(in);
//String line = buff.readLine();
    //System.out.println(line);

//Element head = doc.head();
//Element body = doc.body();
//String linkText = body.text();
//String linkText1 = body.data();

//System.out.println(linkText);
//System.out.println(linkText1);
 
/* while (line != null) {
    text.append(line + "\n");
    line = buff.readLine();
    text1 = Jsoup.parse(line).text();    
    //System.out.println(text1);
} */
//System.out.println((text.noTags(strings)));
//System.out.println(text.toString().replaceAll("\\<[^>]*>",""));


